#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
    Part of Seagull - The Pyhon Pages engine
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

    Python Page Compiler -- Lexer
"""
__author__ = 'Jonathan Slenders <jonathan at slenders dot be>'
__license__ = 'GNU LGPL'
__all__ = ('Lexer', 'Token')

import re

from compile_exceptions import CompileException

class Token(object):
	"""
	Token in the parse tree
	"""
	def __init__(self, name, line=0, column=0):
		self.name = name
		self.line = line
		self.column = column
		self.children = []
		self.attributes = { }

	def append(self, child):
		self.children.append(child)

	def set_attribute(self, name, value):
		self.attributes[name] = value

	def get_string_value(self):
		""" Return all str-childs concatenated """
		result = []
		for c in self.children:
			if isinstance(c, basestring):
				result.append(c)
		return ''.join(result)

	def clone(self):
		"""
		Clone everything within the parse tree.
		None of the nodes in the new tree will have references to the old
		tree
		"""
		# Create new tree
		tree = Token(self.name, self.line, self.column)

		# Copy attributes
		for a in self.attributes:
			tree.set_attribute(a, self.attributes[a])

		# Copy children
		for c in self.children:
			if isinstance(c, basestring):
				tree.append(c)
			else:
				tree.append(c.clone())

		return tree

	def _print(self, prefix=''):
		result = []

		result.append('\033[34m')
		result.append ("%s(%s,%s) {\n" % (self.name, str(self.line), str(self.column)))
		
		for a in self.attributes:
			result.append('%s = %s\n' % (a, self.attributes[a]))

		result.append('\033[0m')

		children_result = []
		for t in self.children:
			if isinstance(t, basestring):
				children_result.append('str(%s)\n' % t)
			else:
				children_result.append("%s\n" % t)
		result.append(''.join(['\t%s\n' % s for s in ''.join(children_result).split('\n')]))

		result.append('\033[34m')
		result.append("}\n")
		result.append('\033[0m')
		return ''.join(result)

	def __str__(self):
		""" Just for debugging the parser """
		return self._print()
			
class State(object):
	"""
	Parse state. Contains a list of regex we my find in the current
	context. Each parse state consists of an ordered list of transitions.
	"""
	class Transition(object):
		def __init__(self, regex_match, action_list):
			"""
			Parse state transition. Consits of a regex
			and an action list that should be executed whet
			this regex has been found.
			"""
			self.regex_match = regex_match
			self.compiled_regex = re.compile(regex_match)
			self.action_list = action_list

	def __init__(self, *transitions):
		self.__transitions = transitions

	def transitions(self):
		""" Transition iterator """
		for t in self.__transitions:
			yield t.compiled_regex, t.action_list

###
# Following classes are action classes for the lexer/parser
###

class ParseAction(object):
	""" Abstract base class, does nothing. """
	pass

class Push(ParseAction):
	"""
	Push this state to the state tack. Parsing
	shall continue by examining this state.
	"""
	def __init__(self, state_name):
		self.state_name = state_name

class Pop(ParseAction):
	"""
	Pop from the state stack.
	"""
	pass

class Record(ParseAction):
	"""
	Record the matched text into the current
	token.
	"""
	def __init__(self, value=None):
		self.value = value

class Shift(ParseAction):
	"""
	Shift the parse pointer after the match.
	"""
	pass

class StartToken(ParseAction):
	"""
	Push this token to the parse stack. New
	tokens or records shall be inserted as
	child of this one.
	"""
	def __init__(self, state_name, **attributes):
		self.state_name = state_name
		self.attributes = attributes

class StopToken(ParseAction):
	"""
	Pop the current token from the parse stack.
	"""
	pass

class SetAttributes(ParseAction):
	"""
	Add meta info to the current token.
	Mostly used for writing back to the original source
	"""
	def __init__(self, **attributes):
		self.attributes = attributes

class Error(ParseAction):
	"""
	Raises an error. We don't expect this match here.
	"""
	def __init__(self, message):
		self.message = message


class Lexer(object):
	"""
	Lexer for a Python Page.
	Unlike usual, this lexer does -- apart from lexing -- also a build a parse tree.
	"""
	# This state dictionary contains all the transitions between states in the parse/lexing process
	STATES = {
		'root' : State(
				State.Transition(r'<!--', (StartToken('xml-comment'), Shift(), Push('xml-comment'))),
				State.Transition(r'\{\$', (Shift(), Push('literal'))),
				State.Transition(r'\{=', (StartToken('python-print-escaped'), Shift(), Push('python'))),
				State.Transition(r'\{(?=[a-z])', (Shift(), Push('specialtag'))),
				State.Transition(r'\{>\s*\}', (StartToken('end-embedded-object'), Shift(), StopToken())),
				State.Transition(r'\{<', (StartToken('start-embedded-object'), Shift(), Push('embedded-object'))),
				State.Transition(r'\{%', (StartToken('python'), Shift(), Push('python'))),
				State.Transition(r'\{', (StartToken('embedded-object'), Shift(), Push('embedded-object'))),
					# NOTE: whitespace (length >= 2) should always be placed in a separate match (for whitespace compression)
				State.Transition(r'\s\s+', (Record(), Shift())),
				State.Transition(r'([^{}\s]|\s(?!\s))+', (Record(), Shift())),
				#State.Transition(r'(<(?![!%\[])|<\[(?!\[)|<!(?!--)|[^<\s]|\s(?!\s))+', (Record(), Shift())),
				State.Transition(r'.|\s', (Error('Error in parser'),)),
			),
		# <%! ...
		'specialtag': State(
				State.Transition(r'page\b\s*', (StartToken('specialtag-page'), Shift(), Pop(), Push('specialtag-content'))),
				State.Transition(r'master\b\s*', (StartToken('specialtag-master'), Shift(), Pop(), Push('specialtag-content'))),
				State.Transition(r'param\b\s*', (StartToken('specialtag-parameter'), Shift(), Pop(), Push('specialtag-content'))),
				State.Transition(r'content\b\s*', (StartToken('specialtag-content'), Shift(), Pop(), Push('specialtag-content'))),
				State.Transition(r'placeholder\b\s*', (StartToken('specialtag-placeholder'), Shift(), Pop(), Push('specialtag-content'))),
				State.Transition(r'meta\b\s*', (StartToken('specialtag-meta'), Shift(), Pop(), Push('specialtag-content'))),
				State.Transition(r'filter\b\s*', (StartToken('specialtag-filter'), Shift(), Pop(), Push('specialtag-content'))),
				State.Transition(r'call\b\s*', (StartToken('specialtag-call'), Shift(), Pop(), Push('specialtag-content'))),
				State.Transition(r'.|\s', (Error('Unknown Special tag'), )),
			),
		'embedded-object': State(
				State.Transition(r'\s*"', (StartToken('include-path'), Shift(), Pop(), Push('embedded-object-2'))),
				State.Transition(r'.|\s', (Error('Variable name in embedded object should be followed by a colon'), )),
			),
		'embedded-object-2': State(
				State.Transition(r'\\', (Shift(), Push('escaped-character'), )),
				State.Transition(r'"', (StopToken(), Shift(), Pop(), Push('specialtag-content'))),
				State.Transition(r'[^"\\]+', (Record(), Shift())),
			),
		# <[[ ... ]]>
		'literal': State(
				State.Transition(r'\$\}', (Shift(), Pop(), )),
				State.Transition(r'[^$]+', (Record(), Shift())),
				State.Transition(r'$(?!\})', (Record(), Shift())),
			),
		# <% ...
		'python': State(
				State.Transition(r'%\\}\s*', (SetAttributes(skipwhitespace='true'), Shift(), StopToken(), Pop())),
				State.Transition(r'[%=]}', (Shift(), StopToken(), Pop())), # For now... should be changed
				State.Transition(r'u?"""', (StartToken('string', stringtype='triple-double-quoted'), Shift(), Push('triple-double-quoted-string'))),
				State.Transition(r'u?r"""', (StartToken('string', stringtype='triple-double-quoted-raw'), Shift(), Push('triple-double-quoted-raw-string'))),

				State.Transition(r"u?'''", (StartToken('string', stringtype='triple-single-quoted'), Shift(), Push('triple-single-quoted-string'))),
				State.Transition(r"u?r'''", (StartToken('string', stringtype='triple-single-quoted'), Shift(), Push('triple-single-quoted-raw-string'))),

				State.Transition(r'u?"', (StartToken('string', stringtype='double-quoted'), Shift(), Push('double-quoted-string'))),
				State.Transition(r'u?r"', (StartToken('string', stringtype='double-quoted-raw'), Shift(), Push('double-quoted-raw-string'))),

				State.Transition(r"u?'", (StartToken('string', stringtype='single-quoted'), Shift(), Push('single-quoted-string'))),
				State.Transition(r"u?r'", (StartToken('string', stringtype='single-quoted-raw'), Shift(), Push('single-quoted-raw-string'))),

				State.Transition(r'#', (StartToken('single-line-comment'), Shift(), Push('single-line-comment'))),
				State.Transition(r'\s+', (StartToken('python-whitespace'), Record(), StopToken(), Shift())),
				State.Transition(r':', (StartToken('colon'), StopToken(), Shift())),
				State.Transition(r';', (StartToken('semi-colon'), StopToken(), Shift())),
				State.Transition(r'([%()\[\]{}])', (Record(), Shift())),
					# Next line may contain any python block that's `copy-ready', that doesn't
					# need precompilation. Like  'var1+var2*9'
				State.Transition(r'''[^\s"':;%()\[\]{}]+''', (Record(), Shift())),
				State.Transition(r'.|\s', (Error('Isn\'t this list complete -> error in parser'),))
			),
		#   #...
		'single-line-comment': State(
				State.Transition(r'[^\n]+', (Record(), Shift())),
				State.Transition(r'\n', (Shift(), StopToken(), Pop())),
			),
		# <!--
		'xml-comment': State(
				State.Transition('--+>', (StopToken(), Shift(), Pop())),
				State.Transition('([^-]|-+(?!(-|>)))+', (Record(), Shift())),
				State.Transition('.|\s', (Error('Expecting end of comment (<!-- ... -->)'),))
			),
		#   ' ...
		'single-quoted-string': State (
				State.Transition(r"""([^'"\n\\]|\\.)+""", (Record(), Shift())),
				State.Transition(r"'", (Shift(), StopToken(), Pop())),
				State.Transition(r'"', (Record(r'\"'), Shift())),
				State.Transition(r'\n', (Error('No newlines allowed in single quoted string'), )),
			),
		'single-quoted-raw-string': State (
				State.Transition(r"""([^'"\n\\])+""", (Record(), Shift())),
				State.Transition(r'\\\\', (Record(r'\\\\'), Shift())),
				State.Transition(r'\\[^\\]', (Record('\\'), Record(), Shift())),
				State.Transition(r"'", (Shift(), StopToken(), Pop())),
				State.Transition(r'"', (Record(r'\"'), Shift())),
				State.Transition(r'\n', (Error('No newlines allowed in single quoted string'))),
			),
		'double-quoted-string': State (
				State.Transition(r"""([^'"\n\\]|\\.)+""", (Record(), Shift())),
				State.Transition(r'"', (Shift(), StopToken(), Pop())),
				State.Transition(r"'", (Record(r"\'"), Shift())),
				State.Transition(r'\n', (Error('No newlines allowed in single quoted string'), )),
			),
		'double-quoted-raw-string': State (
				State.Transition(r"""([^'"\n\\])+""", (Record(), Shift())),
				State.Transition(r'\\\\', (Record(r'\\\\'), Shift())),
				State.Transition(r'\\[^\\]', (Record('\\'), Record(), Shift())),
				State.Transition(r'"', (Shift(), StopToken(), Pop())),
				State.Transition(r"'", (Record(r"\'"), Shift())),
				State.Transition(r'\n', (Error('No newlines allowed in single quoted string'), )),
			),
		'triple-single-quoted-string': State (
				State.Transition(r"""([^'"\\]|\\.)+""", (Record(), Shift())),
				State.Transition(r'"', (Record(r'\"'), Shift())),
				State.Transition(r"'(?!'')", (Record(r"\'"), Shift())),
				State.Transition(r"'''", (Shift(), StopToken(), Pop())),
			),
		'triple-single-quoted-raw-string': State (
				State.Transition(r"""([^'"\\])+""", (Record(), Shift())),
				State.Transition(r'\\\\', (Record(r'\\\\'), Shift())),
				State.Transition(r'\\[^\\]', (Record('\\'), Record(), Shift())),
				State.Transition(r'"', (Record(r'\"'), Shift())),
				State.Transition(r"'(?!'')", (Record(r"\'"), Shift())),
				State.Transition(r"'''", (Shift(), StopToken(), Pop())),
			),
		'triple-double-quoted-string': State (
				State.Transition(r"""([^"'\\]|\\.)+""", (Record(), Shift())),
				State.Transition(r"'", (Record(r"\'"), Shift())),
				State.Transition(r'"(?!"")', (Record(r'\"'), Shift())),
				State.Transition(r'"""', (Shift(), StopToken(), Pop())),
			),
		'triple-double-quoted-raw-string': State (
				State.Transition(r"""([^"'\\])+""", (Record(), Shift())),
				State.Transition(r'\\\\', (Record(r'\\\\'), Shift())),
				State.Transition(r'\\[^\\]', (Record('\\'), Record(), Shift())),
				State.Transition(r'"(?!"")', (Record(r'\"'), Shift())),
				State.Transition(r"'", (Record(r"\'"), Shift())),
				State.Transition(r'"""', (Shift(), StopToken(), Pop())),
			),
		'specialtag-content': State(
				State.Transition(r'\\}\s*', (SetAttributes(skipwhitespace='true'), Shift(), StopToken(), Pop())),
				State.Transition(r'}', (Shift(), StopToken(), Pop())),
				State.Transition(r'\s', (Shift(), )),
				State.Transition(r'[a-z~]', (
					StartToken('specialtag-param-key'),
					Push('specialtag-param-key'))),
				State.Transition(r'.|\s', (Error('Invalid character in tag'),))
			),
		'specialtag-param-key': State(
				State.Transition(r'[a-z1-9-_~]+', (Record(), Shift())),
				State.Transition(r'="', (Shift(), StopToken(),
					Push('specialtag-param-value'), StartToken('specialtag-param-value'))),
				State.Transition(r'.|\s', (Error('Invalid character in tag-key'),)),
			),
		'specialtag-param-value': State(
				State.Transition(r'\\', (Shift(), Push('escaped-character'), )),
				State.Transition(r'([^"\\])+', (Record(), Shift())),
				State.Transition(r'"', (Shift(), StopToken(), Pop(), Pop())),
				State.Transition(r'.|\s', (Error('Invalid character in value'), ))
			),
		'escaped-character': State(
				State.Transition(r'.', (Record(), Shift(), Pop())),
			),
		}

	def __init__(self, string, start_state='root'):
		"""
		Execute the lexer.
		"""
		# State stack
		state_stack = [ start_state ]

		# Parse stack
		tree = Token(start_state, 0, 0)
		token_stack = [ tree ]

		# Parse position
		position = 0
		line = 1
		column = 1

		while position < len(string):
			# Next line is debug code
			#print state_stack[-1]

			for compiled_regex, action_list in Lexer.STATES[ state_stack[-1] ].transitions():
				match = compiled_regex.match(string[position:])
				if match:
					(start, count) = match.span()

					# Read content
					content = string[position : position + count]
					#print "content=%s" % content

					# Execute actions for this match
					for action in action_list:
						if isinstance(action, Record):
							if action.value:
								token_stack[-1].append(action.value)
							else:
								token_stack[-1].append(content)

						elif isinstance(action, Shift):
							position += count
							count = 0

							# Update row/column
							f = content.find('\n')
							while f >= 0:
								line += 1
								column = 1
								content = content[f+1:]
								f = content.find('\n')
							column += len(content)

						elif isinstance(action, Push):
							state_stack.append(action.state_name)

						elif isinstance(action, Pop):
							state_stack.pop()

						elif isinstance(action, StartToken):
							token = Token(action.state_name, line, column)
							token_stack[-1].append(token)
							token_stack.append(token)

							for a in action.attributes:
								token_stack[-1].set_attribute(a, action.attributes[a])

						elif isinstance(action, SetAttributes):
							for a in action.attributes:
								token_stack[-1].set_attribute(a, action.attributes[a])
						
						elif isinstance(action, StopToken):
							token_stack.pop()

						elif isinstance(action, Error):
							raise CompileException(line, column,  action.message)

					break # Out of for

		if token_stack != [ tree ]:
			top = token_stack[-1]
			raise CompileException(top.line, top.column, '%s not terminated' % top.name)
	
		self._lex_tree = tree

	def get_result(self):
		return self._lex_tree

